package org.example.controller;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;

import java.util.List;

//抓取数据
public class FilePipelinePageProcessor implements PageProcessor {
    @Override
    public void process(Page page) {
//        访问黑马的首页
        Html html = page.getHtml();
//        解析首页中所有的链接地址
//     html.css("a","href");
        List<String> allLinks = html.links().all();
//        把链接地址添加到访问队列中
        page.addTargetRequests(allLinks);
//        把页面传递给pipeline  b保存到磁盘
        page.putField("html", html.get());
//
    }

    @Override
    public Site getSite() {
        return Site.me();
    }

    public static void main(String[] args) {
        FilePipeline pipeline = new FilePipeline("D:\\gongzuo\\java-word");
//        创建一个Scheduler
        QueueScheduler scheduler = new QueueScheduler();
//        指定队列使用布隆过滤器来去重
//        BloomFilterDuplicateRemover初始化一个布隆过滤器，参数就是容量
        scheduler.setDuplicateRemover(new BloomFilterDuplicateRemover(10000000));
//        设置代理服务器
//
//        HttpClientDownloader downloader=new HttpClientDownloader();
//        downloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("119.70.40.101",8080)));


//        使用Spider初始化爬虫
        Spider.create(new FilePipelinePageProcessor())
//                .setDownloader(downloader)
//                .setDownloader(new HttpClientDownloader())
//        设置起始的url
                .addUrl("https://read.douban.com/")

//        设置pipeline
                .addPipeline(new ConsolePipeline())
                .addPipeline(pipeline)
//                设置使用的scheduler对象
                .setScheduler(scheduler)
                .start();
//        启动
    }
}
